import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import pandas as pd

# 访问搜狐财经要闻页面
browser = webdriver.Chrome()
browser.maximize_window()
browser.get('https://www.sohu.com/xtopic/TURBd01EVTBOREF4')
time.sleep(3)

# 定位第一个符合条件的<div>标签并单击
browser.find_element(By.CSS_SELECTOR, 'div.header-sort-container > div').click()
time.sleep(3)

# 向下滚动两次页面
for i in range(2):
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    time.sleep(3)
    
# 获取网页源代码
html = browser.page_source
browser.quit()

# 使用BeautifulSoup模块解析HTML代码
soup = BeautifulSoup(html, 'lxml')

# 提取新闻的标题和网址
div_tags = soup.select('div.FeedList div[class$="title"]')
a_tags = soup.select('div.FeedList > div > a')
data = []
for div, a in zip(div_tags, a_tags):
    title = div.get_text().strip()
    url = a.get('href')
    data.append([title, url])

# 整理和导出数据
df = pd.DataFrame(data, columns=['标题', '网址'])
df.to_excel('搜狐财经要闻.xlsx', index=False)
